{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1181,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./mnist/input_data1\\train-images-idx3-ubyte.gz\n",
      "Extracting ./mnist/input_data1\\train-labels-idx1-ubyte.gz\n",
      "Extracting ./mnist/input_data1\\t10k-images-idx3-ubyte.gz\n",
      "Extracting ./mnist/input_data1\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = './mnist/input_data1'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一个非常非常简陋的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1183,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create the model\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "random_seed = 2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1184,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1 = tf.Variable(tf.truncated_normal([784, 784],mean=0,stddev=0.05,seed=random_seed))\n",
    "b1 = tf.Variable(tf.truncated_normal([784],mean=0,stddev=0.05,seed=random_seed))\n",
    "y1 = tf.matmul(x, W1) + b1\n",
    "y1 = tf.nn.relu(y1)\n",
    "\n",
    "W2 = tf.Variable(tf.truncated_normal([784, 300],mean=0,stddev=0.05,seed=random_seed))\n",
    "b2 = tf.Variable(tf.truncated_normal([300],mean=0,stddev=0.05,seed=random_seed))\n",
    "y2 = tf.matmul(y1, W2) + b2\n",
    "y2 = tf.nn.relu(y2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1185,
   "metadata": {},
   "outputs": [],
   "source": [
    "W = tf.Variable(tf.zeros([300, 10]))\n",
    "b = tf.Variable(tf.zeros([10]))\n",
    "y = tf.matmul(y2, W) + b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义我们的ground truth 占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1186,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1187,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "\n",
    "reg_lambda = 0.01 #惩罚因子，值越大，惩罚力度越大\n",
    "regularizer = tf.contrib.layers.l2_regularizer(reg_lambda)\n",
    "regularization = regularizer(W) +  regularizer(b) \n",
    "# 原计划是想要对损失函数加上正则的，结果发现最后的效果反而不如不加，于是最后就没加正则了\n",
    "cross_entropy_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1188,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_step = tf.train.GradientDescentOptimizer(0.7).minimize(cross_entropy_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。\n",
    "然后我们运行3k个step(5 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the loss on train is:2.166237, the accuracy on train is: 0.150000\n",
      "the accuracy on test is:0.10100000351667404\n",
      "\n",
      "the loss on train is:0.019542, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.963699996471405\n",
      "\n",
      "the loss on train is:0.023867, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.9740999937057495\n",
      "\n",
      "the loss on train is:0.005308, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.974399983882904\n",
      "\n",
      "the loss on train is:0.005586, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.9793000221252441\n",
      "\n",
      "the loss on train is:0.020147, the accuracy on train is: 0.990000\n",
      "the accuracy on test is:0.9790999889373779\n",
      "\n",
      "the loss on train is:0.004577, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.9805999994277954\n",
      "\n",
      "the loss on train is:0.001023, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.9818999767303467\n",
      "\n",
      "the loss on train is:0.000358, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.9833999872207642\n",
      "\n",
      "the loss on train is:0.000739, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.983299970626831\n",
      "\n",
      "the loss on train is:0.000432, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.9818000197410583\n",
      "\n",
      "the loss on train is:0.000331, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.9832000136375427\n",
      "\n",
      "the loss on train is:0.000413, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.9843000173568726\n",
      "\n",
      "the loss on train is:0.000462, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.9847999811172485\n",
      "\n",
      "the loss on train is:0.000140, the accuracy on train is: 1.000000\n",
      "the accuracy on test is:0.9854000210762024\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "train_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(train_prediction, tf.float32))\n",
    "\n",
    "test_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(test_prediction, tf.float32))\n",
    "\n",
    "for i in range(10000):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    if i % 500 == 0:\n",
    "        cur_cross_entropy_loss = sess.run(cross_entropy_loss,feed_dict={x: batch_xs, y_: batch_ys})\n",
    "        print('the loss on train is:%f, the accuracy on train is: %f' % (cur_cross_entropy_loss,sess.run(accuracy, feed_dict={x: batch_xs,y_: batch_ys})))\n",
    "        print('the accuracy on test is:{0}\\n'.format(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels})))\n",
    "        if cur_cross_entropy_loss <= 0.0003:\n",
    "            break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证我们模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the accuracy on test is:0.9854000210762024\n",
      "\n"
     ]
    }
   ],
   "source": [
    "  # Test trained model\n",
    "test_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(test_prediction, tf.float32))\n",
    "print('the accuracy on test is:{0}\\n'.format(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels})))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
